# -*- coding: utf-8 -*-
__author__ = 'yixiaoqun'
# -*- coding: utf-8 -*-
import re
import scrapy
import datetime
from scrapy.http import Request
from urllib import parse
from scrapy.loader import ItemLoader
import time
from bole.items import JobBoleArticleItem
from bole.utils.common import get_md5

class JobboleItemSpider(scrapy.Spider):
    name = "jobboleItem"
    allowed_domains = ["python.jobbole.com"]
    start_urls = ['http://python.jobbole.com/all-posts/']

    def parse(self, response):
        """
        1. 获取文章列表页中的文章url并交给scrapy下载后并进行解析
        2. 获取下一页的url并交给scrapy进行下载， 下载完成后交给parse
        """

        #解析列表页中的所有文章url并交给scrapy下载后并进行解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            print("post_url:"+post_url)
            #当读到下一次暂停下，并回调parse_detail函数
            #parse.urljoin如果post_url中有域名，就不连接，如果没有域名，就和当前域名拼接
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url":image_url}, callback=self.parse_detail)

        #提取下一页并交给scrapy进行下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            #拉取下一次，并回调parse函数
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)
        pass
    def parse_detail(self, response):
        article_item=JobBoleArticleItem()


        #提取文章的具体字段
        #通过css选择器提取字段
        front_image_url = response.meta.get("front_image_url", "")  #文章封面图
        print("front_image_url:"+ front_image_url)
        title = response.css(".entry-header h1::text").extract()[0]
        print("title:"+ title)
        create_date = response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace("·","").strip()
        praise_nums = response.css(".vote-post-up h10::text").extract()[0]
        fav_nums = response.css(".bookmark-btn::text").extract()[0]
        match_re = re.match(".*?(\d+).*", fav_nums)
        if match_re:
             fav_nums = int(match_re.group(1))
        else:
            fav_nums = 0
        #
        comment_nums = response.css("a[href='#article-comment'] span::text").extract()[0]
        match_re = re.match(".*?(\d+).*", comment_nums)
        if match_re:
             comment_nums = int(match_re.group(1))
        else:
            comment_nums = 0
        #
        content = response.css("div.entry").extract()[0]
        #
        tag_list = response.css("p.entry-meta-hide-on-mobile a::text").extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

        article_item["title"]=title
        article_item["url"] = response.url
        try:
            create_date = datetime.datetime.strptime(create_date, "%Y/%m/%d").date()
        except Exception as e:
            create_date = datetime.datetime.now().date()
        article_item["create_date"] = create_date
        article_item["url_object_id"] = get_md5(response.url)
        article_item["front_image_url"] = [front_image_url]
        article_item["praise_nums"] =  praise_nums
        article_item["comment_nums"] =comment_nums
        article_item["fav_nums"] = fav_nums
        article_item["tags"] = tags
        article_item["content"] = content
        yield article_item


    #item相当于字典